#!/bin/bash

exec 5>&2
BASH_XTRACEFD="5"

GIT_COMMIT=$(git rev-parse HEAD)
BUILD_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
GIT_BRANCH=${VERSION:-$(git rev-parse --abbrev-ref HEAD | sed -e 's^/^-^g; s^[.]^-^g;' | tr '[:upper:]' '[:lower:]')}
API="pxc.percona.com/v1-9-0"
IMAGE=${IMAGE:-"perconalab/percona-xtradb-cluster-operator:${GIT_BRANCH}"}
IMAGE_PXC=${IMAGE_PXC:-"perconalab/percona-xtradb-cluster-operator:main-pxc8.0"}
IMAGE_PMM=${IMAGE_PMM:-"perconalab/pmm-client:dev-latest"}
IMAGE_PROXY=${IMAGE_PROXY:-"perconalab/percona-xtradb-cluster-operator:main-proxysql"}
IMAGE_HAPROXY=${IMAGE_HAPROXY:-"perconalab/percona-xtradb-cluster-operator:main-haproxy"}
IMAGE_BACKUP=${IMAGE_BACKUP:-"perconalab/percona-xtradb-cluster-operator:main-pxc8.0-backup"}
IMAGE_LOGCOLLECTOR=${IMAGE_LOGCOLLECTOR:-"perconalab/percona-xtradb-cluster-operator:main-logcollector"}
SKIP_BACKUPS_TO_AWS_GCP=${SKIP_BACKUPS_TO_AWS_GCP:-1}
tmp_dir=$(mktemp -d)
sed=$(which gsed || which sed)
date=$(which gdate || which date)

test_name=$(basename $test_dir)
namespace="${test_name}-${RANDOM}"
conf_dir=$(realpath $test_dir/../conf || :)
src_dir=$(realpath $test_dir/../..)

if [ -f "$conf_dir/cloud-secret.yml" ]; then
	SKIP_BACKUPS_TO_AWS_GCP=''
fi

if oc get projects; then
	# Guessing Openshift version from master node API version
	case "$(oc get nodes --no-headers=true | grep master | head -n1 | grep -Eo 'v[0-9]+\.[0-9]+')" in
		"v1.11") OPENSHIFT=3.11 ;;
		*) OPENSHIFT=4 ;;
	esac
fi

KUBE_VERSION=$(kubectl version -o json | jq -r '.serverVersion.major + "." + .serverVersion.minor' | $sed -r 's/[^0-9.]+//g')

HELM_VERSION=$(helm version -c | $sed -re 's/.*SemVer:"([^"]+)".*/\1/; s/.*\bVersion:"([^"]+)".*/\1/')
if [ "${HELM_VERSION:0:2}" == "v2" ]; then
	HELM_ARGS="--name"
fi

version_gt() {
	# return true if kubernetes version equal or greater than desired
	if [ $(echo "${KUBE_VERSION} >= $1" | bc -l) -eq 1 ]; then
		return 0
	else
		return 1
	fi
}

wait_cluster_consistency() {
	local cluster_name=${1}
	local cluster_size=${2}
	local proxy_size=${3}

	if [ -z "${proxy_size}" ]; then
		proxy_size=${cluster_size}
	fi

	sleep 7 # wait for two reconcile loops ;)  3 sec x 2 times + 1 sec = 7 seconds
	until [[ "$(kubectl_bin get pxc "${cluster_name}" -o jsonpath='{.status.state}')" == "ready" &&
	"$(kubectl_bin get pxc "${cluster_name}" -o jsonpath='{.status.pxc.ready}')" == "${cluster_size}" &&
	"$(kubectl_bin get pxc "${cluster_name}" -o jsonpath='{.status.'$(get_proxy_engine ${cluster_name})'.ready}')" == "${proxy_size}" ]]; do
		echo 'waiting for cluster readyness'
		sleep 20
	done
}

create_namespace() {
	local namespace="$1"
	local skip_clean_namespace="$2"

	if [[ ${CLEAN_NAMESPACE} == 1 ]] && [[ -z ${skip_clean_namespace} ]]; then
		kubectl_bin get ns \
			| egrep -v "^kube-|^default|Terminating|pxc-operator|openshift|^NAME" \
			| awk '{print$1}' \
			| xargs kubectl delete ns &
	fi

	if [ ! -z "$OPENSHIFT" ]; then
		oc delete project "$namespace" && sleep 40 || :
		oc new-project "$namespace"
		oc project "$namespace"
		oc adm policy add-scc-to-user hostaccess -z default || :
	else
		kubectl_bin delete namespace "$namespace" || :
		wait_for_delete "namespace/$namespace"
		kubectl_bin create namespace "$namespace"
		kubectl_bin config set-context $(kubectl_bin config current-context) --namespace="$namespace"
	fi
}

get_operator_pod() {
	local label_prefix="app.kubernetes.io/"
	local check_label=$(kubectl get pods --selector=app.kubernetes.io/name=percona-xtradb-cluster-operator ${OPERATOR_NS:+-n $OPERATOR_NS} | grep -c "percona-xtradb-cluster-operator")
	if [[ ${check_label} -eq 0 ]]; then
		label_prefix=""
	fi
	kubectl_bin get pods \
		--selector=${label_prefix}name=percona-xtradb-cluster-operator \
		-o 'jsonpath={.items[].metadata.name}' ${OPERATOR_NS:+-n $OPERATOR_NS}
}

wait_pod() {
	local pod=$1
	local max_retry="${2:-480}"
	local ns=$3
	local container=$(echo "$pod" | $sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' | egrep "^(pxc|proxysql)$")

	set +o xtrace
	retry=0
	echo -n $pod
	until kubectl_bin get ${ns:+-n $ns} pod/$pod -o jsonpath='{.status.conditions[?(@.type == "Ready")].status}' 2>/dev/null | grep -q -i 'True' \
		&& kubectl_bin get ${ns:+-n $ns} pod/$pod | grep -q "^$pod" \
		&& ! kubectl_bin logs --tail=1 ${ns:+-n $ns} pod/$pod ${container:+ -c $container} | grep -q LAST_LINE; do
		sleep 1
		echo -n .
		let retry+=1
		if [[ $retry -ge $max_retry ]]; then
			kubectl_bin describe pod/$pod
			kubectl_bin logs $pod
			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod)
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	echo ".Ok"
	set -o xtrace
}

wait_crash_pod() {
	local pod=$1
	local max_retry="${2:-480}"
	local ns=$3
	local container=$(echo "$pod" | $sed -E 's/.*-(pxc|proxysql)-[0-9]/\1/' | egrep "^(pxc|proxysql)$")

	set +o xtrace
	retry=0
	echo -n $pod
	until kubectl_bin get ${ns:+-n $ns} pod/$pod -o jsonpath='{.status.conditions[?(@.type == "Ready")].status}' 2>/dev/null | grep -q -i 'True' \
		&& kubectl_bin get ${ns:+-n $ns} pod/$pod | grep -q "^$pod" \
		&& kubectl_bin logs --tail=1 ${ns:+-n $ns} pod/$pod ${container:+ -c $container} | grep -q LAST_LINE; do
		sleep 1
		echo -n .
		let retry+=1
		if [[ $retry -ge $max_retry ]]; then
			kubectl_bin describe pod/$pod
			kubectl_bin logs $pod
			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod)
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	echo ".Ok"
	set -o xtrace
}

wait_backup() {
	local backup=$1

	set +o xtrace
	retry=0
	echo -n $backup
	until kubectl_bin get pxc-backup/$backup -o jsonpath='{.status.state}' 2>/dev/null | grep 'Succeeded'; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge 120 ]; then
			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod)
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	set -o xtrace
}

wait_backup_restore() {
	local backup_name=$1

	set +o xtrace
	retry=0
	echo -n $backup_name
	until kubectl_bin get pxc-restore/$backup_name -o jsonpath='{.status.state}' 2>/dev/null | grep 'Succeeded'; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge 1500 ]; then
			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod)
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	echo
	set -o xtrace
}

apply_rbac() {
	local operator_namespace=${OPERATOR_NS:-'pxc-operator'}
	local rbac=${1:-'rbac'}

	cat ${src_dir}/deploy/${rbac}.yaml \
		| sed -e "s^namespace: .*^namespace: $operator_namespace^" \
		| kubectl_bin apply -f -
}

deploy_operator() {
	desc 'start operator'

	kubectl_bin apply -f ${src_dir}/deploy/crd.yaml || :

	if [ -n "$OPERATOR_NS" ]; then
		apply_rbac cw-rbac
		cat ${src_dir}/deploy/cw-operator.yaml \
			| sed -e "s^image: .*^image: ${IMAGE}^" \
			| sed -e "s^failureThreshold: .*^failureThreshold: 10^" \
			| kubectl_bin apply -f -
	else
		apply_rbac rbac
		cat ${src_dir}/deploy/operator.yaml \
			| sed -e "s^image: .*^image: ${IMAGE}^" \
			| sed -e "s^failureThreshold: .*^failureThreshold: 10^" \
			| kubectl_bin apply -f -
	fi

	sleep 10

	wait_pod "$(get_operator_pod)" "480" "${OPERATOR_NS}"
	sleep 3
}

deploy_helm() {
	helm repo add hashicorp https://helm.releases.hashicorp.com
	helm repo add percona https://percona-charts.storage.googleapis.com/
	helm repo add minio https://helm.min.io/
	helm repo update
}

create_infra() {
	local ns="$1"

	if [ -n "$OPERATOR_NS" ]; then
		kubectl get pxc --all-namespaces -o wide \
			| grep -v 'NAMESPACE' \
			| xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' \
			|| :
		kubectl_bin delete pxc --all --all-namespaces || :
		kubectl_bin delete pxc-backup --all --all-namespaces || :
		kubectl_bin delete pxc-restore --all --all-namespaces || :

		create_namespace $OPERATOR_NS
		deploy_operator
		create_namespace $ns
	else
		create_namespace $ns
		deploy_operator
	fi
	apply_secrets
}

wait_for_running() {
	local name="$1"
	let last_pod="$(($2 - 1))" || :
	local max_retry="${3:-480}"

	for i in $(seq 0 $last_pod); do
		wait_pod ${name}-${i} ${max_retry}
	done
}

wait_for_delete() {
	local res="$1"

	set +o xtrace
	echo -n "$res - "
	retry=0
	until (kubectl_bin get $res || :) 2>&1 | grep NotFound; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge 120 ]; then
			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod)
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	set -o xtrace
}

compare_kubectl() {
	local resource="$1"
	local postfix="$2"
	local expected_result=${test_dir}/compare/${resource//\//_}${postfix}.yml
	local new_result="${tmp_dir}/${resource//\//_}.yml"

	if [ ! -z "$OPENSHIFT" -a -f ${expected_result//.yml/-oc.yml} ]; then
		expected_result=${expected_result//.yml/-oc.yml}
		if [ "$OPENSHIFT" = 4 -a -f ${expected_result//-oc.yml/-4-oc.yml} ]; then
			expected_result=${expected_result//-oc.yml/-4-oc.yml}
		fi
	fi

	if [[ $IMAGE_PXC =~ 8\.0 ]] && [ -f ${expected_result//.yml/-80.yml} ]; then
		expected_result=${expected_result//.yml/-80.yml}
	fi

	kubectl_bin get -o yaml ${resource} \
		| yq d - 'metadata.managedFields' \
		| yq d - '**.creationTimestamp' \
		| yq d - '**.namespace' \
		| yq d - '**.uid' \
		| yq d - 'metadata.resourceVersion' \
		| yq d - 'metadata.selfLink' \
		| yq d - 'metadata.deletionTimestamp' \
		| yq d - 'metadata.annotations."k8s.v1.cni.cncf.io*"' \
		| yq d - 'metadata.annotations."kubernetes.io/psp"' \
		| yq d - '**.(name==percona-xtradb-cluster-operator-workload-token*)' \
		| yq d - '**.creationTimestamp' \
		| yq d - '**.image' \
		| yq d - '**.clusterIP' \
		| yq d - '**.clusterIPs' \
		| yq d - '**.dataSource' \
		| yq d - '**.procMount' \
		| yq d - '**.storageClassName' \
		| yq d - '**.finalizers' \
		| yq d - '**."kubernetes.io/pvc-protection"' \
		| yq d - '**.volumeName' \
		| yq d - '**."volume.beta.kubernetes.io/storage-provisioner"' \
		| yq d - 'spec.volumeMode' \
		| yq d - 'spec.nodeName' \
		| yq d - '**."volume.kubernetes.io/selected-node"' \
		| yq d - '**."percona.com/*"' \
		| yq d - '**.(volumeMode==Filesystem).volumeMode' \
		| yq d - '**.healthCheckNodePort' \
		| yq d - '**.nodePort' \
		| yq d - '**.imagePullSecrets' \
		| yq d - '**.enableServiceLinks' \
		| yq d - 'status' \
		| yq d - '**.(name==NAMESPACE)' \
		| yq d - '**.(name==suffix)' \
		| yq d - '**.(name==S3_BUCKET_PATH)' \
		| yq d - '**.(name==S3_BUCKET_URL)' \
		| yq d - 'spec.volumeClaimTemplates.*.apiVersion' \
		| yq d - 'spec.volumeClaimTemplates.*.kind' \
		| yq d - 'metadata.ownerReferences.*.apiVersion' \
		| yq d - '**.controller-uid' \
		| yq d - '**.preemptionPolicy' \
		| sed 's/namespace\:.*name/name/' \
			>${new_result}
	# last sed is needed for backup cronjob content

	diff -u ${expected_result} ${new_result}
}

get_client_pod() {
	kubectl_bin get pods \
		--selector=name=pxc-client \
		-o 'jsonpath={.items[].metadata.name}'
}

run_mysql() {
	local command="$1"
	local uri="$2"

	client_pod=$(get_client_pod)
	wait_pod $client_pod 1>&2

	[[ ${-/x/} != $- ]] && echo "+ kubectl exec -it $client_pod -- bash -c \"printf '$command\n' | mysql -sN $uri\"" >&$BASH_XTRACEFD

	set +o xtrace
	kubectl_bin exec $client_pod -- \
		bash -c "printf '%s\n' \"${command}\" | mysql -sN $uri" 2>&1 \
		| sed -e 's/mysql: //' \
		| (grep -v 'Using a password on the command line interface can be insecure.' || :)
	set -o xtrace
}

run_mysql_local() {
	local command="$1"
	local uri="$2"
	local pod="$3"
	local container_name="$4"

	[[ ${-/x/} != $- ]] && echo "+ kubectl exec -it $pod -- bash -c \"printf '$command\n' | mysql -sN $uri\"" >&$BASH_XTRACEFD

	set +o xtrace
	kubectl_bin exec $pod $container_name -- \
		bash -c "printf '$command\n' | mysql -sN $uri" 2>&1 \
		| sed -e 's/mysql: //' \
		| (egrep -v 'Using a password on the command line interface can be insecure.|Defaulted container|Defaulting container name|see all of the containers in this pod' || :)
	set -o xtrace
}

compare_mysql_cmd() {
	local command_id="$1"
	local command="$2"
	local uri="$3"
	local postfix="$4"
	local expected_result=${test_dir}/compare/${command_id}${postfix}.sql

	if [[ $IMAGE_PXC =~ 8\.0 ]] && [ -f ${test_dir}/compare/${command_id}${postfix}-80.sql ]; then
		expected_result=${test_dir}/compare/${command_id}${postfix}-80.sql
	fi

	run_mysql "$command" "$uri" \
		>$tmp_dir/${command_id}.sql
	if [ ! -s "$tmp_dir/${command_id}.sql" ]; then
		sleep 20
		run_mysql "$command" "$uri" \
			>$tmp_dir/${command_id}.sql
	fi
	diff -u $expected_result $tmp_dir/${command_id}.sql
}

compare_mysql_cmd_local() {
	local command_id="$1"
	local command="$2"
	local uri="$3"
	local pod="$4"
	local postfix="$5"
	local container_name="$6"
	local expected_result=${test_dir}/compare/${command_id}${postfix}.sql

	if [[ $IMAGE_PXC =~ 8\.0 ]] && [ -f ${test_dir}/compare/${command_id}${postfix}-80.sql ]; then
		expected_result=${test_dir}/compare/${command_id}${postfix}-80.sql
	fi

	run_mysql_local "$command" "$uri" "$pod" "$container_name" \
		>$tmp_dir/${command_id}.sql
	if [ ! -s "$tmp_dir/${command_id}.sql" ]; then
		sleep 20
		run_mysql_local "$command" "$uri" "$pod" \
			>$tmp_dir/${command_id}.sql
	fi
	diff -u $expected_result $tmp_dir/${command_id}.sql
}

get_proxy_primary() {
	local uri="$1"
	local pod="$2"
	local ip=$(run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' "$uri" "$pod" 'proxysql')

	while [ $(echo "$ip" | wc -l) != 1 ]; do
		sleep 1
		ip=$(run_mysql_local 'SELECT hostname FROM runtime_mysql_servers WHERE hostgroup_id=11 AND status="ONLINE";' "$uri" "$pod" 'proxysql')
	done

	echo $ip | cut -d'.' -f1
}

get_pod_name() {
	local ip=$1
	kubectl_bin get pods -o json | jq -r '.items[] | select(.status.podIP == "'$ip'") | .metadata.name'
}

get_pod_ip() {
	local name=$1
	kubectl_bin get pods -o json | jq -r '.items[] | select(.metadata.name == "'$name'") | .status.podIP'
}

compare_mysql_user() {
	local uri="$1"
	local postfix="$2"
	local user=$(echo $uri | sed -e 's/.*-u//; s/ .*//')
	local expected_result=${test_dir}/compare/$user$postfix.sql

	if [[ $IMAGE_PXC =~ 8\.0 ]] && [ -f ${test_dir}/compare/$user$postfix-80.sql ]; then
		expected_result=${test_dir}/compare/$user$postfix-80.sql
	fi

	(run_mysql "SHOW GRANTS;" "$uri" || :) \
		| $sed -E "s/'(10|192)[.][0-9][^']*'//; s/'[^']*[.]internal'//" \
			>$tmp_dir/$user.sql
	diff -u $expected_result $tmp_dir/$user.sql
}

compare_mysql_user_local() {
	local uri="$1"
	local pod="$2"
	local postfix="$3"
	local container_name="$4"
	local user=$(echo $uri | sed -e 's/.*-u//; s/ .*//')
	local expected_result=$test_dir/compare/$user$postfix.sql

	if [[ $IMAGE_PXC =~ 8\.0 ]] && [ -f ${test_dir}/compare/$user$postfix-80.sql ]; then
		expected_result=${test_dir}/compare/$user$postfix-80.sql
	fi

	(run_mysql_local "SHOW GRANTS;" "$uri" "$pod" "$container_name" || :) \
		| $sed -E "s/'(10|192)[.][0-9][^']*'//; s/'[^']*[.]internal'//" \
			>$tmp_dir/$user.sql
	diff -u $expected_result $tmp_dir/$user.sql
}

get_pumba() {
	kubectl_bin get pods \
		--selector=name=pumba \
		-o 'jsonpath={.items[].metadata.name}'
}

run_pumba() {
	local cmd="$*"
	kubectl_bin exec -it "$(get_pumba)" -- /pumba -l info ${cmd}
}

deploy_cert_manager() {
	kubectl_bin create namespace cert-manager || :
	kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true || :
	kubectl_bin apply -f https://github.com/jetstack/cert-manager/releases/download/v0.15.1/cert-manager.yaml --validate=false || : 2>/dev/null
	sleep 45
}

destroy() {
	local namespace="$1"

	kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
		| grep -v '"level":"info"' \
		| grep -v 'the object has been modified' \
		| grep -v 'get backup status: Job.batch' \
		| $sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' \
		| sort -u \
		| tee $tmp_dir/operator.log

	#TODO: maybe will be enabled later
	#diff $test_dir/compare/operator.log $tmp_dir/operator.log

	kubectl get pxc --all-namespaces -o wide \
		| grep -v 'NAMESPACE' \
		| xargs -L 1 sh -xc 'kubectl patch pxc -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' \
		|| :
	kubectl_bin delete pxc --all --all-namespaces || :
	kubectl_bin delete pxc-backup --all --all-namespaces || :
	kubectl_bin delete pxc-restore --all --all-namespaces || :
	kubectl_bin delete ValidatingWebhookConfiguration percona-xtradbcluster-webhook || :

	kubectl_bin delete -f https://github.com/jetstack/cert-manager/releases/download/v0.15.1/cert-manager.yaml 2>/dev/null || :
	if [ ! -z "$OPENSHIFT" ]; then
		oc delete --grace-period=0 --force=true project "$namespace" &
		if [ -n "$OPERATOR_NS" ]; then
			oc delete --grace-period=0 --force=true project "$OPERATOR_NS" &
			if [ "${OPENSHIFT}" == "3.11" ]; then
				wait_for_delete "project/$OPERATOR_NS" # 3.11 Does not delete namespace immediately, let's wait at least 120 seconds till the next step
			fi
		fi
	else
		kubectl_bin delete --grace-period=0 --force=true namespace "$namespace" &
		if [ -n "$OPERATOR_NS" ]; then
			kubectl_bin delete --grace-period=0 --force=true namespace "$OPERATOR_NS" &
		fi
	fi
	rm -rf ${tmp_dir}
}

desc() {
	set +o xtrace
	local msg="$@"
	printf "\n\n-----------------------------------------------------------------------------------\n"
	printf "$msg"
	printf "\n-----------------------------------------------------------------------------------\n\n"
	set -o xtrace
}

get_service_endpoint() {
	local service=$1

	local hostname=$(
		kubectl_bin get service/$service -o json \
			| jq '.status.loadBalancer.ingress[].hostname' \
			| sed -e 's/^"//; s/"$//;'
	)
	if [ -n "$hostname" -a "$hostname" != "null" ]; then
		echo $hostname
		return
	fi

	local ip=$(
		kubectl_bin get service/$service -o json \
			| jq '.status.loadBalancer.ingress[].ip' \
			| sed -e 's/^"//; s/"$//;'
	)
	if [ -n "$ip" -a "$ip" != "null" ]; then
		echo $ip
		return
	fi

	exit 1
}

get_metric_values() {
	local metric=$1
	local instance=$2
	local user_pass=$3
	local start=$($date -u "+%s" -d "-1 minute")
	local end=$($date -u "+%s")
	local endpoint=$(get_service_endpoint monitoring-service)

	curl -s -k "https://${user_pass}@$endpoint/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28$metric%7Bnode_name%3D%7E%22$instance%22%7d%20or%20$metric%7Bnode_name%3D%7E%22$instance%22%7D%29&start=$start&end=$end&step=60" \
		| jq '.data.result[0].values[][1]' \
		| grep '^"[0-9]'

}

get_qan_values() {
	local instance=$1
	local start=$($date -u "+%Y-%m-%dT%H:%M:%S" -d "-30 minute")
	local end=$($date -u "+%Y-%m-%dT%H:%M:%S")
	local endpoint=$(get_service_endpoint monitoring-service)

	local uuid=$(
		curl -s -k "https://$endpoint/qan-api/instances?deleted=no" \
			| jq '.[] | select(.Subsystem == "mysql" and .Name == "'$instance'") | .UUID' \
			| sed -e 's/^"//; s/"$//;'
	)

	curl -s -k "https://$endpoint/qan-api/qan/profile/$uuid?begin=$start&end=$end&offset=0" \
		| jq '.Query[].Fingerprint'
}

get_qan20_values() {
	local instance=$1
	local user_pass=$2
	local start=$($date -u "+%Y-%m-%dT%H:%M:%S" -d "-30 minute")
	local end=$($date -u "+%Y-%m-%dT%H:%M:%S")
	local endpoint=$(get_service_endpoint monitoring-service)

	cat >payload.json <<EOF
{
   "columns":[
      "load",
      "num_queries",
      "query_time"
   ],
   "first_seen": false,
   "group_by": "queryid",
   "include_only_fields": [],
   "keyword": "",
   "labels": [
       {
           "key": "cluster",
           "value": ["pxc"]
   }],
   "limit": 10,
   "offset": 0,
   "order_by": "-load",
   "main_metric": "load",
   "period_start_from": "$($date -u -d '-12 hour' '+%Y-%m-%dT%H:%M:%S%:z')",
   "period_start_to": "$($date -u '+%Y-%m-%dT%H:%M:%S%:z')"
}
EOF

	curl -s -k -XPOST -d @payload.json "https://${user_pass}@$endpoint/v0/qan/GetReport" \
		| jq '.rows[].fingerprint'
	rm -f payload.json
}

cat_config() {
	cat "$1" \
		| $sed -e "s#apiVersion: pxc.percona.com/v.*\$#apiVersion: $API#" \
		| $sed -e "s#image:.*-pxc\([0-9]*.[0-9]*\)\{0,1\}\$#image: $IMAGE_PXC#" \
		| $sed -e "s#image:.*\/percona-xtradb-cluster:.*\$#image: $IMAGE_PXC#" \
		| $sed -e "s#image:.*-pmm\$#image: $IMAGE_PMM#" \
		| $sed -e "s#image:.*-backup\$#image: $IMAGE_BACKUP#" \
		| $sed -e "s#image:.*-proxysql\$#image: $IMAGE_PROXY#" \
		| $sed -e "s#image:.*-haproxy\$#image: $IMAGE_HAPROXY#" \
		| $sed -e "s#image:.*-logcollector\$#image: $IMAGE_LOGCOLLECTOR#" \
		| $sed -e "s~minio-service.#namespace~minio-service.$namespace~" \
		| $sed -e "s#apply:.*#apply: Never#"
}

apply_secrets() {
	if [ -z "$SKIP_BACKUPS_TO_AWS_GCP" ]; then
		kubectl_bin apply \
			-f $conf_dir/minio-secret.yml \
			-f $conf_dir/cloud-secret.yml
	else
		kubectl_bin apply \
			-f $conf_dir/minio-secret.yml
	fi
}

apply_config() {
	if [ -z "$SKIP_BACKUPS_TO_AWS_GCP" ]; then
		cat_config "$1" \
			| kubectl_bin apply -f -
	else
		cat_config "$1" \
			| yq d - 'spec.backup.schedule.[1]' \
			| yq d - 'spec.backup.schedule.[2]' \
			| kubectl_bin apply -f -
	fi
}

get_proxy() {
	local target_cluster=${1}
	if [[ "$(kubectl_bin get pxc ${target_cluster} -o 'jsonpath={.spec.haproxy.enabled}')" == "true" ]]; then
		echo "${target_cluster}-haproxy"
		return
	fi
	if [[ "$(kubectl_bin get pxc ${target_cluster} -o 'jsonpath={.spec.proxysql.enabled}')" == "true" ]]; then
		echo "${target_cluster}-proxysql"
		return
	fi
	echo "${target_cluster}-pxc"
}

get_proxy_engine() {
	local cluster_name=$1
	local cluster_proxy="$(get_proxy ${cluster_name})"
	echo "${cluster_proxy//$cluster_name-/}"
}

spinup_pxc() {
	local cluster=$1
	local config=$2
	local size="${3:-3}"
	local sleep="${4:-10}"
	local secretsFile="${5:-$conf_dir/secrets.yml}"
	local pxcClientFile="${6:-$conf_dir/client.yml}"

	desc 'create first PXC cluster'
	kubectl_bin apply -f $secretsFile
	apply_config "$pxcClientFile"
	if [[ $IMAGE_PXC =~ 5\.7 ]] && [ "$cluster" == 'demand-backup' ]; then
		cat_config "$config" \
			| $sed '/\[sst\]/,+1d' \
			| $sed 's|compress=lz4|compress|' \
			| kubectl_bin apply -f -
	else
		apply_config "$config"
	fi

	desc 'check if all 3 Pods started'
	local proxy=$(get_proxy "$cluster")
	wait_for_running "$proxy" 1
	wait_for_running "$cluster-pxc" "$size"
	sleep $sleep

	desc 'write data'
	if [[ $IMAGE_PXC =~ 5\.7 ]] && [[ "$(is_keyring_plugin_in_use "$cluster")" ]]; then
		encrypt='ENCRYPTION=\"Y\"'
	fi
	run_mysql \
		"CREATE DATABASE IF NOT EXISTS myApp; use myApp; CREATE TABLE IF NOT EXISTS myApp (id int PRIMARY KEY) $encrypt;" \
		"-h $proxy -uroot -proot_password"
	run_mysql \
		'INSERT myApp.myApp (id) VALUES (100500)' \
		"-h $proxy -uroot -proot_password"
	sleep 30
	for i in $(seq 0 $((size - 1))); do
		compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-$i.$cluster-pxc -uroot -proot_password"
	done

	if [ "$(is_keyring_plugin_in_use "$cluster")" ]; then
		table_must_be_encrypted "$cluster" "myApp"
	fi

}

function is_table_encrypted() {
	local cluster=$1
	local table=$2
	run_mysql \
		"SELECT CREATE_OPTIONS FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME=\\\"$table\\\";" \
		"-h $cluster-proxysql -uroot -proot_password" \
		| egrep -o "ENCRYPTION=('Y'|\"Y\")"
}

function is_keyring_plugin_in_use() {
	local cluster=$1
	kubectl_bin exec -it $cluster-pxc-0 -c pxc -- bash -c "cat /etc/mysql/node.cnf" \
		| egrep -o "early-plugin-load=keyring_\w+.so"
}

function table_must_not_be_encrypted() {
	local cluster=$1
	local table=$2
	if is_table_encrypted "$cluster" "$table"; then
		echo "error: table is encrypted"
		exit 1
	fi
}

function table_must_be_encrypted() {
	local cluster=$1
	local table=$2
	if ! is_table_encrypted "$cluster" "$table"; then
		echo "error: table is not encrypted"
		exit 1
	fi
}

function keyring_plugin_must_be_in_use() {
	local cluster=$1
	if ! is_keyring_plugin_in_use "$cluster"; then
		echo "error: keyring_plugin is not used"
		exit 1
	fi
}

function keyring_plugin_must_not_be_in_use() {
	local cluster=$1
	if is_keyring_plugin_in_use "$cluster"; then
		echo "error: keyring_plugin is used"
		exit 1
	fi
}

kubectl_bin() {
	local LAST_OUT="$(mktemp)"
	local LAST_ERR="$(mktemp)"
	local exit_status=0
	for i in $(seq 0 2); do
		kubectl "$@" 1>"$LAST_OUT" 2>"$LAST_ERR"
		exit_status=$?
		[[ ${-/x/} != $- ]] && echo "--- $i stdout" | cat - "$LAST_OUT" >&$BASH_XTRACEFD
		[[ ${-/x/} != $- ]] && echo "--- $i stderr" | cat - "$LAST_ERR" >&$BASH_XTRACEFD
		if [[ ${exit_status} != 0 ]]; then
			sleep "$((timeout * i))"
		else
			cat "$LAST_OUT"
			cat "$LAST_ERR" >&2
			rm "$LAST_OUT" "$LAST_ERR"
			return ${exit_status}
		fi
	done
	cat "$LAST_OUT"
	cat "$LAST_ERR" >&2
	rm "$LAST_OUT" "$LAST_ERR"
	return ${exit_status}
}

retry() {
	local max=$1
	local delay=$2
	shift 2 # cut delay and max args
	local n=1

	until "$@"; do
		if [[ $n -ge $max ]]; then
			echo "The command '$@' has failed after $n attempts."
			exit 1
		fi
		((n++))
		sleep $delay
	done
}

check_pvc_md5() {
	desc 'check backup file md5sum'
	apply_config "$test_dir/conf/client.yml"
	sleep 10
	bak_client_pod=$(
		kubectl_bin get pods \
			--selector=name=backup-client \
			-o 'jsonpath={.items[].metadata.name}'
	)
	wait_pod $bak_client_pod
	kubectl_bin exec $bak_client_pod -- \
		bash -c "cd /backup; md5sum -c md5sum.txt"
	kubectl_bin delete \
		-f $test_dir/conf/client.yml
}

run_backup() {
	local cluster=$1
	local backup1=$2

	desc 'make backup'
	kubectl_bin apply \
		-f $test_dir/conf/$backup1.yml
	wait_backup $backup1
}

run_recovery_check() {
	local cluster=$1
	local backup1=$2

	desc 'recover backup'
	kubectl_bin apply -f "$test_dir/conf/restore-${backup1}.yaml"
	wait_backup_restore ${backup1}
	kubectl_bin logs job/restore-job-${backup1}-${cluster}
	kubectl_bin delete -f "$test_dir/conf/restore-${backup1}.yaml"
	wait_for_running "$cluster-proxysql" 1
	wait_for_running "$cluster-pxc" 3

	sleep 35
	desc 'check data after backup'
	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-0.$cluster-pxc -uroot -proot_password"
	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-1.$cluster-pxc -uroot -proot_password"
	compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h $cluster-pxc-2.$cluster-pxc -uroot -proot_password"

	if [ "$backup1" != "on-demand-backup-minio" ]; then
		desc 'copy backup'
		bash $src_dir/deploy/backup/copy-backup.sh $backup1 $tmp_dir/backup
	fi
}

function vault_tls() {
	local name=${1:-vault-service}

	SERVICE=$name
	NAMESPACE=$name
	SECRET_NAME=$name
	CSR_NAME=vault-csr-${RANDOM}

	openssl genrsa -out ${tmp_dir}/vault.key 2048
	cat <<EOF >${tmp_dir}/csr.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = ${SERVICE}
DNS.2 = ${SERVICE}.${NAMESPACE}
DNS.3 = ${SERVICE}.${NAMESPACE}.svc
DNS.4 = ${SERVICE}.${NAMESPACE}.svc.cluster.local
IP.1 = 127.0.0.1
EOF

	openssl req -new -key ${tmp_dir}/vault.key -subj "/CN=${SERVICE}.${NAMESPACE}.svc" -out ${tmp_dir}/server.csr -config ${tmp_dir}/csr.conf

	cat <<EOF >${tmp_dir}/csr.yaml
apiVersion: certificates.k8s.io/v1beta1
kind: CertificateSigningRequest
metadata:
  name: ${CSR_NAME}
spec:
  groups:
  - system:authenticated
  request: $(cat ${tmp_dir}/server.csr | base64 | tr -d '\n')
  usages:
  - digital signature
  - key encipherment
  - server auth
EOF

	kubectl_bin create -f ${tmp_dir}/csr.yaml
	sleep 10
	kubectl_bin certificate approve ${CSR_NAME}
	kubectl_bin get csr ${CSR_NAME} -o jsonpath='{.status.certificate}' >${tmp_dir}/serverCert
	openssl base64 -in ${tmp_dir}/serverCert -d -A -out ${tmp_dir}/vault.crt
	kubectl_bin config view --raw --minify --flatten -o jsonpath='{.clusters[].cluster.certificate-authority-data}' | base64 -d >${tmp_dir}/vault.ca
	if [[ -n ${OPENSHIFT} ]]; then
		if [[ "x$(kubectl_bin get namespaces | awk '{print $1}' | grep openshift-kube-controller-manager-operator)" != "x" ]]; then
			#Detecting openshift 4+
			kubectl_bin -n openshift-kube-controller-manager-operator get secret csr-signer -o jsonpath='{.data.tls\.crt}' \
				| base64 -d >${tmp_dir}/vault.ca
		else
			CA_SECRET_NAME=$(kubectl_bin -n default get secrets \
				| grep default \
				| grep service-account-token \
				| head -n 1 \
				| awk {'print $1'})
			kubectl_bin -n default get secret ${CA_SECRET_NAME} -o jsonpath='{.data.ca\.crt}' \
				| base64 -d >${tmp_dir}/vault.ca
		fi
	fi
	kubectl_bin create secret generic ${SECRET_NAME} \
		--namespace ${NAMESPACE} \
		--from-file=vault.key=${tmp_dir}/vault.key \
		--from-file=vault.crt=${tmp_dir}/vault.crt \
		--from-file=vault.ca=${tmp_dir}/vault.ca
}

start_vault() {
	name=${1:-vault-service}
	protocol=${2:-http}

	local platform=kubernetes

	if [[ -n ${OPENSHIFT} ]]; then
		platform=openshift
		oc patch clusterrole system:auth-delegator --type='json' -p '[{"op":"add","path":"/rules/-", "value":{"apiGroups":["security.openshift.io"], "attributeRestrictions":null, "resourceNames": ["privileged"], "resources":["securitycontextconstraints"],"verbs":["use"]}}]'
	fi
	create_namespace "$name" "skip_clean"
	deploy_helm "$name"
	helm delete "$name" || :

	desc "install Vault $name"

	if [ $protocol == "https" ]; then
		vault_tls "$name"
		helm install $name hashicorp/vault \
			--disable-openapi-validation \
			--version 0.6.0 \
			--namespace "$name" \
			--set dataStorage.enabled=false \
			--set global.tlsDisable=false \
			--set global.platform="${platform}" \
			--set server.extraVolumes[0].type=secret \
			--set server.extraVolumes[0].name=$name \
			--set server.extraEnvironmentVars.VAULT_CACERT=/vault/userconfig/$name/vault.ca \
			--set server.standalone.config="
listener \"tcp\" {
    address = \"[::]:8200\"
    cluster_address = \"[::]:8201\"
    tls_cert_file = \"/vault/userconfig/$name/vault.crt\"
    tls_key_file  = \"/vault/userconfig/$name/vault.key\"
    tls_client_ca_file = \"/vault/userconfig/$name/vault.ca\"
}

storage \"file\" {
    path = \"/vault/data\"
}"

	else
		helm install $name hashicorp/vault \
			--disable-openapi-validation \
			--version 0.6.0 \
			--namespace "$name" \
			--set dataStorage.enabled=false \
			--set global.platform="${platform}"
	fi

	if [[ -n ${OPENSHIFT} ]]; then
		oc patch clusterrole $name-agent-injector-clusterrole --type='json' -p '[{"op":"add","path":"/rules/-", "value":{"apiGroups":["security.openshift.io"], "attributeRestrictions":null, "resourceNames": ["privileged"], "resources":["securitycontextconstraints"],"verbs":["use"]}}]'
		oc adm policy add-scc-to-user privileged $name-agent-injector
	fi

	set +o xtrace
	retry=0
	echo -n pod/$name-0
	until kubectl_bin get pod/$name-0 -o 'jsonpath={.status.containerStatuses[0].state}' 2>/dev/null | grep 'running'; do
		echo -n .
		sleep 1
		let retry+=1
		if [ "$retry" -ge 480 ]; then
			kubectl_bin describe pod/$name-0
			kubectl_bin logs $name-0
			echo max retry count "$retry" reached. something went wrong with vault
			exit 1
		fi
	done
	set -o xtrace

	kubectl_bin exec -it $name-0 -- vault operator init -tls-skip-verify -key-shares=1 -key-threshold=1 -format=json >"$tmp_dir/$name"
	unsealKey=$(jq -r ".unseal_keys_b64[]" <"$tmp_dir/$name")
	token=$(jq -r ".root_token" <"$tmp_dir/$name")
	sleep 10

	kubectl_bin exec -it $name-0 -- vault operator unseal -tls-skip-verify "$unsealKey"
	kubectl_bin exec -it $name-0 -- \
		sh -c "export VAULT_TOKEN=$token && export VAULT_LOG_LEVEL=trace \
                && vault secrets enable --version=1 -tls-skip-verify -path=secret kv \
                && vault audit enable file file_path=/vault/vault-audit.log"
	sleep 10

	cat "$conf_dir/vault-secret.yaml" \
		| sed -e "s/#token/$token/" \
		| sed -e "s/#vault_url/$protocol:\/\/$name.$name.svc.cluster.local:8200/" \
		| sed -e "s/#secret/secret/" >"${tmp_dir}/vault-secret.yaml"
	if [ $protocol == "https" ]; then
		sed -e 's/^/    /' ${tmp_dir}/vault.ca >${tmp_dir}/vault.new.ca
		$sed -i "s/#vault_ca/vault_ca/" "${tmp_dir}/vault-secret.yaml"
		$sed -i "/#certVal/r ${tmp_dir}/vault.new.ca" "${tmp_dir}/vault-secret.yaml"
		$sed -i "/#certVal/d" "${tmp_dir}/vault-secret.yaml"
	else
		$sed -i "/#vault_ca/d" "${tmp_dir}/vault-secret.yaml"
	fi

	kubectl_bin apply --namespace="$namespace" -f ${tmp_dir}/vault-secret.yaml

	kubectl_bin config set-context "$(kubectl_bin config current-context)" --namespace="$namespace"
}

start_minio() {
	deploy_helm $namespace

	desc 'install Minio'
	helm del minio-service || :
	retry 10 60 helm install \
		$HELM_ARGS \
		minio-service \
		--version 8.0.5 \
		--set accessKey=some-access-key \
		--set secretKey=some-secret-key \
		--set service.type=ClusterIP \
		--set configPathmc=/tmp/.minio/ \
		--set securityContext.enabled=false \
		--set persistence.size=2G \
		--set environment.MINIO_REGION=us-east-1 \
		--set environment.MINIO_HTTP_TRACE=/tmp/trace.log \
		minio/minio
	sleep 30
	MINIO_POD=$(kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}')
	wait_pod $MINIO_POD

	kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
		/usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
		/usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing
}

deploy_chaos_mesh() {
	local chaos_mesh_ns=$1

	desc 'install chaos-mesh'
	local old_cm_namespace=$(helm list --all-namespaces --filter chaos-mesh | tail -n1 | awk -F' ' '{print $2}')
	if [ "${old_cm_namespace}" != "NAMESPACE" ]; then
		helm del chaos-mesh --namespace ${old_cm_namespace} || :
	fi
	helm repo add chaos-mesh https://charts.chaos-mesh.org
	if version_gt "1.19"; then
		helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=${chaos_mesh_ns} --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version 0.5.1 --set clusterScoped=false --set controllerManager.targetNamespace=${chaos_mesh_ns}
	else
		helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=${chaos_mesh_ns} --set dashboard.create=false --version 0.5.1 --set clusterScoped=false --set controllerManager.targetNamespace=${chaos_mesh_ns}
	fi
	sleep 10
}

destroy_chaos_mesh() {
	local chaos_mesh_ns=$1

	desc 'destroy chaos-mesh'
	helm del chaos-mesh --namespace ${chaos_mesh_ns} || :
}

patch_secret() {
	local secret=$1
	local key=$2
	local value=$3

	kubectl_bin patch secret $secret -p="{\"data\":{\"$key\": \"$value\"}}"
}

getSecretData() {
	local secretName=$1
	local dataKey=$2
	kubectl_bin get secrets/${secretName} --template={{.data.${dataKey}}} \
		| base64 --decode
}

checkTLSSecret() {
	local secretName=$1
	local dataKey=$2
	local secretData=$(kubectl_bin get secrets/${secretName} -o json | jq '.data["'${dataKey}'"]')
	if [ -z "$secretData" ]; then
		exit 1
	fi
}

tlsSecretsShouldExist() {
	local secretName=$1
	checkTLSSecret "$secretName" 'ca.crt'
	checkTLSSecret "$secretName" 'tls.crt'
	checkTLSSecret "$secretName" 'tls.key'
}

function check_pxc_liveness() {
	local cluster="$1"
	local cluster_size="$2"
	wait_cluster_consistency "${cluster}" "${cluster_size}"

	wait_for_running "${cluster}-pxc" "${cluster_size}"

	for i in $(seq 0 $((cluster_size - 1))); do
		compare_mysql_cmd "select-1" "SELECT * from myApp.myApp;" "-h ${cluster}-pxc-${i}.${cluster}-pxc -uroot -proot_password"
	done
}

function compare_generation() {
	local generation="$1"
	local proxy="$2"
	local cluster="$3"
	local current_generation

	if [[ ${proxy} == "haproxy" ]]; then
		containers=(pxc haproxy)
	else
		containers=(pxc proxysql)
	fi
	for container in "${containers[@]}"; do
		current_generation="$(kubectl_bin get statefulset "${cluster}-${container}" -o jsonpath='{.metadata.generation}')"
		if [[ ${generation} != "${current_generation}" ]]; then
			echo "Generation for resource ${container} is: ${current_generation}, but should be: ${generation}"
			exit 1
		fi
	done
}

function apply_rbac_gh() {
	local operator_namespace="${OPERATOR_NS:-'pxc-operator'}"
	local rbac="${1:-'rbac'}"
	local git_tag="$2"

	curl -s "https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/${git_tag}/deploy/${rbac}.yaml" >"${tmp_dir}/rbac_${git_tag}.yaml"
	$sed -i -e "s^namespace: .*^namespace: ${operator_namespace}^" "${tmp_dir}/rbac_${git_tag}.yaml"
	kubectl_bin apply -f "${tmp_dir}/rbac_${git_tag}.yaml"
}

function deploy_operator_gh() {
	local git_tag="$1"

	desc 'start operator'
	if [[ -n $(kubectl_bin get crds -o jsonpath='{.items[?(@.metadata.name == "perconaxtradbclusters.pxc.percona.com")].metadata.name}') ]] \
		&& [[ -n $(kubectl_bin get crd/perconaxtradbclusters.pxc.percona.com -o jsonpath='{.spec.versions[?(@.name == "'"${git_tag//\./-}"'")].name}') ]]; then
		echo "Target CRD for ${git_tag} is in place"
	else
		kubectl_bin apply -f "https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/${git_tag}/deploy/crd.yaml" >"${tmp_dir}/crd_${git_tag}.yaml"
	fi

	local rbac_yaml="rbac"
	local operator_yaml="operator.yaml"
	if [ -n "${OPERATOR_NS}" ]; then
		rbac_yaml="cw-rbac"
		operator_yaml="cw-operator.yaml"
	fi
	apply_rbac_gh "${rbac_yaml}" "${git_tag}"
	curl -s "https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/${git_tag}/deploy/${operator_yaml}" >"${tmp_dir}/${operator_yaml}_${git_tag}.yaml"
	$sed -i -e "s^image: .*^image: ${IMAGE}^" "${tmp_dir}/${operator_yaml}_${git_tag}.yaml"
	kubectl_bin apply -f "${tmp_dir}/${operator_yaml}_${git_tag}.yaml" ${OPERATOR_NS:+-n $OPERATOR_NS}

	sleep 2
	wait_pod "$(get_operator_pod)"
}

function create_infra_gh() {
	local ns="$1"
	local git_tag="$2"

	if [ -n "${OPERATOR_NS}" ]; then
		create_namespace "${OPERATOR_NS}"
		deploy_operator_gh "${git_tag}"
		create_namespace "${ns}"
	else
		create_namespace "${ns}"
		deploy_operator_gh "${git_tag}"
	fi
	apply_secrets
}

function prepare_cr_yaml() {
	local cr_yaml="$1"
	local proxy="$2"
	local cluster="$3"
	local cluster_size="$4"
	local git_tag="$5"

	# spinup function expects images to have suffix like "-pxc"
	# to replace them with images from environment variables
	curl -s "https://raw.githubusercontent.com/percona/percona-xtradb-cluster-operator/${git_tag}/deploy/cr.yaml" \
		| yq w - "metadata.name" "${cluster}" \
		| yq w - "spec.secretsName" "my-cluster-secrets" \
		| yq w - "spec.vaultSecretName" "some-name-vault" \
		| yq w - "spec.sslSecretName" "some-name-ssl" \
		| yq w - "spec.sslInternalSecretName" "some-name-ssl-internal" \
		| yq w - "spec.upgradeOptions.apply" "disabled" \
		| yq w - "spec.pxc.size" "${cluster_size}" \
		| yq w - "spec.proxysql.size" "${cluster_size}" \
		| yq w - "spec.haproxy.size" "${cluster_size}" \
		| yq w - "spec.pxc.image" -- "-pxc" \
		| yq w - "spec.proxysql.image" -- "-proxysql" \
		| yq w - "spec.haproxy.image" -- "-haproxy" \
		| yq w - "spec.backup.image" -- "-backup" >"${cr_yaml}"
	if [[ ${proxy} == "haproxy" ]]; then
		yq w -i "${cr_yaml}" "spec.haproxy.enabled" "true"
		yq w -i "${cr_yaml}" "spec.proxysql.enabled" "false"
	else
		yq w -i "${cr_yaml}" "spec.haproxy.enabled" "false"
		yq w -i "${cr_yaml}" "spec.proxysql.enabled" "true"
	fi
}
